# Load API key and secret from environment variables
from dotenv import load_dotenv
load_dotenv()
# ML libraries
import pandas as pd
import xgboost as xgb
from numpy import argmax
from sklearn.metrics import accuracy_score, precision_recall_curve
from sklearn.model_selection import train_test_split
# ValidMind libraries
from sklearn.metrics import accuracy_score, precision_recall_curve
from sklearn.model_selection import train_test_split
import validmind as vm
from validmind.vm_models.test_context import TestContext
# Plotting libraries
import matplotlib.pyplot as plt
%matplotlib inlineCRE/MF Mortgage Rate Model Demo
Setup
Load Data
df = pd.read_csv("../datasets/lending_club_loan_rates.csv", sep='\t')
df = df.rename(columns={'Unnamed: 0': 'Date'})
df = df.set_index(pd.to_datetime(df['Date']))
df.drop(["Date"], axis=1, inplace=True)
# Remove diff columns
columns_to_remove = [col for col in df.columns if col.startswith("diff")]
df = df.drop(columns=columns_to_remove)
df.head()| loan_rate_A | loan_rate_B | loan_rate_C | loan_rate_D | FEDFUNDS | |
|---|---|---|---|---|---|
| Date | |||||
| 2007-08-01 | 7.766667 | 9.497692 | 10.947500 | 12.267000 | 5.02 |
| 2007-09-01 | 7.841429 | 9.276667 | 10.829167 | 12.436667 | 4.94 |
| 2007-10-01 | 7.830000 | 9.433333 | 10.825926 | 12.737368 | 4.76 |
| 2007-11-01 | 7.779091 | 9.467778 | 10.967037 | 12.609444 | 4.49 |
| 2007-12-01 | 7.695833 | 9.387500 | 10.805000 | 12.478889 | 4.24 |
Visual Inspection.
ValidMind Setup
Initialize ValidMind dataset.
vm.init(
api_host = "http://localhost:3000/api/v1/tracking",
api_key = "e22b89a6b9c2a27da47cb0a09febc001",
api_secret = "a61be901b5596e3c528d94231e4a3c504ef0bb803d16815f8dfd6857fac03e57",
project = "clgo0g0rt0000fjy6ozl9pb69"
)True
target_variables = ["loan_rate_A", "loan_rate_B", "loan_rate_C", "loan_rate_D"]
vm_dataset = vm.init_dataset(
dataset=df,
target_column = target_variables
)Pandas dataset detected. Initializing VM Dataset instance...
Inferring dataset types...
Visualize existing test plans.
vm.test_plans.list_plans()4. Model Development
4.1. Development Data and Platform
4.1.2. Data Quality and Relevance
4.1.3. Data Process, Adjustments and Treatment
A. Missing Values Analysis
Step 1: Calculate the percentage of missing values in each column
Step 2: Display the missing values percentage in a table format
Step 3: Visualize the missing values
### B. Outliers Analysis
Step 1: Visualize the dataset using box plots
Visualize the data using box plots to get an initial sense of the presence of outliers.
Step 2: Calculate Z-scores
Step 3: Set a threshold and identify outliers
Set a threshold (e.g., 3) to identify data points with Z-scores higher than the threshold.
Step 4: Analyze the outliers
Analyze the outliers by looking at their frequency, index, and corresponding column.
### C. Stationarity Analysis
Step 1: Run Unit Root Tests
from validmind.test_plans.statsmodels_timeseries import UnitRoot
test_context = TestContext(dataset=vm_dataset)
ur_test_plan = UnitRoot(test_context=test_context)
ur_test_plan.run()Running Metric: kpss: 20%|██ | 1/5 [00:00<00:00, 89.93it/s] The test statistic is outside of the range of p-values available in the
look-up table. The actual p-value is smaller than the p-value returned.
The test statistic is outside of the range of p-values available in the
look-up table. The actual p-value is greater than the p-value returned.
The test statistic is outside of the range of p-values available in the
look-up table. The actual p-value is smaller than the p-value returned.
The test statistic is outside of the range of p-values available in the
look-up table. The actual p-value is smaller than the p-value returned.
Results for unit_root Test Plan:
Logged the following dataset metric to the ValidMind platform:
{'loan_rate_A': {'stat': -1.917289312690944, 'pvalue': 0.32397189281015515, 'usedlag': 1, 'nobs': 135, 'critical_values': {'1%': -3.479742586699182, '5%': -2.88319822181578, '10%': -2.578319684499314}, 'icbest': -71.08908853191068}, 'loan_rate_B': {'stat': -3.1599303710498425, 'pvalue': 0.022424413263559147, 'usedlag': 9, 'nobs': 127, 'critical_values': {'1%': -3.482920063655088, '5%': -2.884580323367261, '10%': -2.5790575441750883}, 'icbest': -42.45027033820841}, 'loan_rate_C': {'stat': -2.530666699941385, 'pvalue': 0.10818994357289696, 'usedlag': 1, 'nobs': 135, 'critical_values': {'1%': -3.479742586699182, '5%': -2.88319822181578, '10%': -2.578319684499314}, 'icbest': -92.19465856666866}, 'loan_rate_D': {'stat': -1.617158531178829, 'pvalue': 0.47421928207593467, 'usedlag': 6, 'nobs': 130, 'critical_values': {'1%': -3.4816817173418295, '5%': -2.8840418343195267, '10%': -2.578770059171598}, 'icbest': -4.9426661983780775}, 'FEDFUNDS': {'stat': -0.16854321128256927, 'pvalue': 0.9421687822974046, 'usedlag': 13,...
Logged the following evaluation metric to the ValidMind platform:
{'loan_rate_A': {'stat': 1.012356679488042, 'pvalue': 0.01, 'usedlag': 6, 'critical_values': {'10%': 0.347, '5%': 0.463, '2.5%': 0.574, '1%': 0.739}}, 'loan_rate_B': {'stat': 0.26307336980308743, 'pvalue': 0.1, 'usedlag': 6, 'critical_values': {'10%': 0.347, '5%': 0.463, '2.5%': 0.574, '1%': 0.739}}, 'loan_rate_C': {'stat': 0.8099610581510324, 'pvalue': 0.01, 'usedlag': 6, 'critical_values': {'10%': 0.347, '5%': 0.463, '2.5%': 0.574, '1%': 0.739}}, 'loan_rate_D': {'stat': 1.5641456258111677, 'pvalue': 0.01, 'usedlag': 6, 'critical_values': {'10%': 0.347, '5%': 0.463, '2.5%': 0.574, '1%': 0.739}}, 'FEDFUNDS': {'stat': 0.37574599218114024, 'pvalue': 0.08760948612881886, 'usedlag': 6, 'critical_values': {'10%': 0.347, '5%': 0.463, '2.5%': 0.574, '1%': 0.739}}}
Logged the following evaluation metric to the ValidMind platform:
{'loan_rate_A': {'stat': -2.027461766308415, 'pvalue': 0.2746633686983735, 'usedlag': 13, 'nobs': 136}, 'loan_rate_B': {'stat': -2.4460840425020693, 'pvalue': 0.1291495662057986, 'usedlag': 13, 'nobs': 136}, 'loan_rate_C': {'stat': -2.264992296812233, 'pvalue': 0.18350727828330493, 'usedlag': 13, 'nobs': 136}, 'loan_rate_D': {'stat': -1.8536131757387562, 'pvalue': 0.35418744654020773, 'usedlag': 13, 'nobs': 136}, 'FEDFUNDS': {'stat': -3.970771700595016, 'pvalue': 0.00157153429138061, 'usedlag': 13, 'nobs': 136}}
Logged the following evaluation metric to the ValidMind platform:
{'loan_rate_A': {'stat': -3.4986701764941164, 'pvalue': 0.6804771233224802, 'usedlag': 1, 'nobs': 137}, 'loan_rate_B': {'stat': -5.24634607418534, 'pvalue': 0.011857344006006859, 'usedlag': 9, 'nobs': 137}, 'loan_rate_C': {'stat': -4.681009266047663, 'pvalue': 0.07413460165891156, 'usedlag': 10, 'nobs': 137}, 'loan_rate_D': {'stat': -4.5661357909991125, 'pvalue': 0.10001302102995041, 'usedlag': 5, 'nobs': 137}, 'FEDFUNDS': {'stat': -4.263682850153341, 'pvalue': 0.20816298878228218, 'usedlag': 13, 'nobs': 137}}
Logged the following evaluation metric to the ValidMind platform:
{'loan_rate_A': {'stat': -1.79800835619108, 'pvalue': 0.07127713927441981, 'usedlag': 1, 'nobs': 135}, 'loan_rate_B': {'stat': -1.3606975485926263, 'pvalue': 0.16677847263199563, 'usedlag': 9, 'nobs': 127}, 'loan_rate_C': {'stat': -0.4700035237018868, 'pvalue': 0.5125489900593104, 'usedlag': 10, 'nobs': 126}, 'loan_rate_D': {'stat': 0.2675751876186429, 'pvalue': 0.777908700165412, 'usedlag': 5, 'nobs': 131}, 'FEDFUNDS': {'stat': -1.4167584741514279, 'pvalue': 0.15109358273536777, 'usedlag': 13, 'nobs': 123}}
Unit Root Tests with Stationarity Decision.
# Question: Ideally we would like to show results of unit root test plan like this (results hardcoded)
unit_root_test_results = {
'Series': ['loan_rate_A', 'loan_rate_A', 'loan_rate_A', 'loan_rate_A', 'loan_rate_A', 'loan_rate_B', 'loan_rate_B', 'loan_rate_B', 'loan_rate_B', 'loan_rate_B', 'loan_rate_C', 'loan_rate_C', 'loan_rate_C', 'loan_rate_C', 'loan_rate_C', 'loan_rate_D', 'loan_rate_D', 'loan_rate_D', 'loan_rate_D', 'loan_rate_D', 'FEDFUNDS', 'FEDFUNDS', 'FEDFUNDS', 'FEDFUNDS', 'FEDFUNDS'],
'Test': ['ADF', 'KPSS', 'Phillips-Perron', 'Zivot-Andrews', 'DFGLS', 'ADF', 'KPSS', 'Phillips-Perron', 'Zivot-Andrews', 'DFGLS', 'ADF', 'KPSS', 'Phillips-Perron', 'Zivot-Andrews', 'DFGLS', 'ADF', 'KPSS', 'Phillips-Perron', 'Zivot-Andrews', 'DFGLS', 'ADF', 'KPSS', 'Phillips-Perron', 'Zivot-Andrews', 'DFGLS'],
'p-value': [0.323972, 0.010000, 0.274663, 0.680477, 0.071277, 0.022424, 0.100000, 0.129150, 0.011857, 0.166778, 0.108190, 0.010000, 0.183507, 0.074135, 0.512549, 0.474219, 0.010000, 0.354187, 0.100013, 0.777909, 0.942169, 0.087609, 0.001572, 0.208163, 0.151094],
'Threshold': [0.05] * 25,
'Pass/Fail': ['Pass', 'Pass', 'Pass', 'Pass', 'Pass', 'Fail', 'Fail', 'Pass', 'Fail', 'Pass', 'Pass', 'Pass', 'Pass', 'Pass', 'Pass', 'Pass', 'Pass', 'Pass', 'Pass', 'Pass', 'Pass', 'Fail', 'Fail', 'Pass', 'Pass'],
'Decision': ['non-stationary', 'stationary', 'non-stationary', 'non-stationary', 'non-stationary', 'stationary', 'non-stationary', 'non-stationary', 'stationary', 'non-stationary', 'non-stationary', 'stationary', 'non-stationary', 'non-stationary', 'non-stationary', 'non-stationary', 'stationary', 'non-stationary', 'non-stationary', 'non-stationary', 'non-stationary', 'non-stationary', 'stationary', 'non-stationary', 'non-stationary']
}
display(pd.DataFrame(unit_root_test_results))| Series | Test | p-value | Threshold | Pass/Fail | Decision | |
|---|---|---|---|---|---|---|
| 0 | loan_rate_A | ADF | 0.323972 | 0.05 | Pass | non-stationary |
| 1 | loan_rate_A | KPSS | 0.010000 | 0.05 | Pass | stationary |
| 2 | loan_rate_A | Phillips-Perron | 0.274663 | 0.05 | Pass | non-stationary |
| 3 | loan_rate_A | Zivot-Andrews | 0.680477 | 0.05 | Pass | non-stationary |
| 4 | loan_rate_A | DFGLS | 0.071277 | 0.05 | Pass | non-stationary |
| 5 | loan_rate_B | ADF | 0.022424 | 0.05 | Fail | stationary |
| 6 | loan_rate_B | KPSS | 0.100000 | 0.05 | Fail | non-stationary |
| 7 | loan_rate_B | Phillips-Perron | 0.129150 | 0.05 | Pass | non-stationary |
| 8 | loan_rate_B | Zivot-Andrews | 0.011857 | 0.05 | Fail | stationary |
| 9 | loan_rate_B | DFGLS | 0.166778 | 0.05 | Pass | non-stationary |
| 10 | loan_rate_C | ADF | 0.108190 | 0.05 | Pass | non-stationary |
| 11 | loan_rate_C | KPSS | 0.010000 | 0.05 | Pass | stationary |
| 12 | loan_rate_C | Phillips-Perron | 0.183507 | 0.05 | Pass | non-stationary |
| 13 | loan_rate_C | Zivot-Andrews | 0.074135 | 0.05 | Pass | non-stationary |
| 14 | loan_rate_C | DFGLS | 0.512549 | 0.05 | Pass | non-stationary |
| 15 | loan_rate_D | ADF | 0.474219 | 0.05 | Pass | non-stationary |
| 16 | loan_rate_D | KPSS | 0.010000 | 0.05 | Pass | stationary |
| 17 | loan_rate_D | Phillips-Perron | 0.354187 | 0.05 | Pass | non-stationary |
| 18 | loan_rate_D | Zivot-Andrews | 0.100013 | 0.05 | Pass | non-stationary |
| 19 | loan_rate_D | DFGLS | 0.777909 | 0.05 | Pass | non-stationary |
| 20 | FEDFUNDS | ADF | 0.942169 | 0.05 | Pass | non-stationary |
| 21 | FEDFUNDS | KPSS | 0.087609 | 0.05 | Fail | non-stationary |
| 22 | FEDFUNDS | Phillips-Perron | 0.001572 | 0.05 | Fail | stationary |
| 23 | FEDFUNDS | Zivot-Andrews | 0.208163 | 0.05 | Pass | non-stationary |
| 24 | FEDFUNDS | DFGLS | 0.151094 | 0.05 | Pass | non-stationary |
Interpretation of Unit Root Tests.
Step 2: Making Series Stationary
Compute first difference.
diff_df = df.diff().dropna()Inspect time series.
Step 3: Run Unit Root Tests
# Pass first difference to VM dataset
# Question: I am now overwriting the df, can I log both raw and first diff dataset and use them as required later on?
vm_dataset = vm.init_dataset(dataset=diff_df)
test_context = TestContext(dataset=vm_dataset)
ur_test_plan = UnitRoot(test_context=test_context)
ur_test_plan.run()Pandas dataset detected. Initializing VM Dataset instance...
Inferring dataset types...
Running Metric: kpss: 20%|██ | 1/5 [00:00<00:00, 82.73it/s] The test statistic is outside of the range of p-values available in the
look-up table. The actual p-value is greater than the p-value returned.
The test statistic is outside of the range of p-values available in the
look-up table. The actual p-value is greater than the p-value returned.
The test statistic is outside of the range of p-values available in the
look-up table. The actual p-value is greater than the p-value returned.
The test statistic is outside of the range of p-values available in the
look-up table. The actual p-value is greater than the p-value returned.
The test statistic is outside of the range of p-values available in the
look-up table. The actual p-value is smaller than the p-value returned.
Results for unit_root Test Plan:
Logged the following dataset metric to the ValidMind platform:
{'loan_rate_A': {'stat': -10.288173999889596, 'pvalue': 3.628391608895891e-18, 'usedlag': 0, 'nobs': 135, 'critical_values': {'1%': -3.479742586699182, '5%': -2.88319822181578, '10%': -2.578319684499314}, 'icbest': -77.77997210900043}, 'loan_rate_B': {'stat': -8.581774306100574, 'pvalue': 7.693931170347862e-14, 'usedlag': 0, 'nobs': 135, 'critical_values': {'1%': -3.479742586699182, '5%': -2.88319822181578, '10%': -2.578319684499314}, 'icbest': -39.517450997610695}, 'loan_rate_C': {'stat': -8.497490203712452, 'pvalue': 1.264293522721377e-13, 'usedlag': 0, 'nobs': 135, 'critical_values': {'1%': -3.479742586699182, '5%': -2.88319822181578, '10%': -2.578319684499314}, 'icbest': -89.85288082240504}, 'loan_rate_D': {'stat': -4.089047184282974, 'pvalue': 0.0010096960791906875, 'usedlag': 5, 'nobs': 130, 'critical_values': {'1%': -3.4816817173418295, '5%': -2.8840418343195267, '10%': -2.578770059171598}, 'icbest': -2.8530273746351327}, 'FEDFUNDS': {'stat': -7.7635076306973865, 'pvalue': 9.31881384691925e-12, 'usedla...
Logged the following evaluation metric to the ValidMind platform:
{'loan_rate_A': {'stat': 0.08690727348877204, 'pvalue': 0.1, 'usedlag': 2, 'critical_values': {'10%': 0.347, '5%': 0.463, '2.5%': 0.574, '1%': 0.739}}, 'loan_rate_B': {'stat': 0.1406230557198794, 'pvalue': 0.1, 'usedlag': 3, 'critical_values': {'10%': 0.347, '5%': 0.463, '2.5%': 0.574, '1%': 0.739}}, 'loan_rate_C': {'stat': 0.302111183227079, 'pvalue': 0.1, 'usedlag': 3, 'critical_values': {'10%': 0.347, '5%': 0.463, '2.5%': 0.574, '1%': 0.739}}, 'loan_rate_D': {'stat': 0.2150982513758707, 'pvalue': 0.1, 'usedlag': 1, 'critical_values': {'10%': 0.347, '5%': 0.463, '2.5%': 0.574, '1%': 0.739}}, 'FEDFUNDS': {'stat': 1.1100238305291938, 'pvalue': 0.01, 'usedlag': 5, 'critical_values': {'10%': 0.347, '5%': 0.463, '2.5%': 0.574, '1%': 0.739}}}
Logged the following evaluation metric to the ValidMind platform:
{'loan_rate_A': {'stat': -10.31157577193113, 'pvalue': 3.175252114879247e-18, 'usedlag': 13, 'nobs': 135}, 'loan_rate_B': {'stat': -8.656685376413767, 'pvalue': 4.9471754291376904e-14, 'usedlag': 13, 'nobs': 135}, 'loan_rate_C': {'stat': -8.968716395365107, 'pvalue': 7.859548454989337e-15, 'usedlag': 13, 'nobs': 135}, 'loan_rate_D': {'stat': -9.359989759533976, 'pvalue': 7.870234405243031e-16, 'usedlag': 13, 'nobs': 135}, 'FEDFUNDS': {'stat': -5.577757930833498, 'pvalue': 1.4193167380557158e-06, 'usedlag': 13, 'nobs': 135}}
Logged the following evaluation metric to the ValidMind platform:
{'loan_rate_A': {'stat': -10.647554732355527, 'pvalue': 1e-05, 'usedlag': 0, 'nobs': 136}, 'loan_rate_B': {'stat': -8.878000651641708, 'pvalue': 1e-05, 'usedlag': 0, 'nobs': 136}, 'loan_rate_C': {'stat': -9.176489623666406, 'pvalue': 1e-05, 'usedlag': 0, 'nobs': 136}, 'loan_rate_D': {'stat': -4.7362602346101195, 'pvalue': 0.06385037699908429, 'usedlag': 5, 'nobs': 136}, 'FEDFUNDS': {'stat': -8.74079625000198, 'pvalue': 1e-05, 'usedlag': 13, 'nobs': 136}}
Logged the following evaluation metric to the ValidMind platform:
{'loan_rate_A': {'stat': -9.407796046868018, 'pvalue': 8.038704892435365e-16, 'usedlag': 0, 'nobs': 135}, 'loan_rate_B': {'stat': -6.1286593160527945, 'pvalue': 6.978825245356776e-09, 'usedlag': 0, 'nobs': 135}, 'loan_rate_C': {'stat': -6.7890425662958975, 'pvalue': 2.863415572617712e-10, 'usedlag': 0, 'nobs': 135}, 'loan_rate_D': {'stat': -3.2170698858406395, 'pvalue': 0.0013827003089759758, 'usedlag': 5, 'nobs': 130}, 'FEDFUNDS': {'stat': -3.775689552643456, 'pvalue': 0.0001896667660634073, 'usedlag': 13, 'nobs': 122}}
Unit Root Tests with Stationarity Decision.
Interpretation of Unit Root Tests.
Step 4: Decision
Series is stationary after first difference.
D. Seasonality Analysis
Step 1: Seasonal decomposition
Perform seasonal decomposition on each time series.
from validmind.model_validation.statsmodels.metrics import SeasonalDecompose
test_context = TestContext(train_ds=vm_train_ds)
sd_metric = SeasonalDecompose(test_context=test_context)NameError: name 'vm_train_ds' is not defined
Step 2: Visualize seasonal decomposition
Create plots for observed, trend, seasonal and residual components.
sd_metric.run()
sd_metric.result.show()Seasonality Detection using ACF and PACF.
from validmind.model_validation.statsmodels.metrics import SeasonalityDetectionWithACFandPACF
test_context = TestContext(train_ds=vm_train_ds)
acf_metric = SeasonalityDetectionWithACFandPACF(test_context=test_context)
acf_metric.run()
acf_metric.result.show()Step 3: Residuals Analysis
Residuals series, histogram, Q-Q and ACF plots.
# Comment: How do I pass the residuals of seasonal decomponsition done before using SeasonalDecomposeMetricWithFigure?
from validmind.model_validation.statsmodels.metrics import ResidualsVisualInspection
test_context = TestContext(train_ds=vm_train_ds)
rvi_metric = ResidualsVisualInspection(test_context=test_context)
rvi_metric.run()rvi_metric.result.show()Test if Residuals are Normaly Distributed.
# Comment: How do I pass the residuals of seasonal decomponsition done before using SeasonalDecomposeMetricWithFigure?
vm.run_test_plan("normality_test_plan", train_ds=vm_train_ds, test_ds=vm_test_ds)Test if Residuals are Autocorrelated.
# Comment: How do I pass the residuals of seasonal decomponsition done before using SeasonalDecomposeMetricWithFigure?
vm.run_test_plan("autocorrelation_test_plan", train_ds=vm_train_ds, test_ds=vm_test_ds)Step 4: Test for seasonality using the Augmented Dickey-Fuller (ADF) test
Step 5: Analyze the seasonality test results
Step 6: Interpret the results
Step 7: Handle seasonality
4.2. Methodology Selection and Development
4.2.4 Variable Analysis
## A. Feature Analysis
A.1. Univariate Analysis
Visual Inspection
A.2 Multivariave Analysis
Visual Inspection
B. Variable Selection
ARIMA Analysis
Step 1: Identify the Integration order (Stationarity Analysis)
Unit Root Tests.
vm.run_test_plan("unit_root_test_plan", train_ds=vm_train_ds, test_ds=vm_test_ds)Step 2: Identify the AR order
Step 3: Identify the MA order
vm.run_test_plan("normality_test_plan", train_ds=vm_train_ds, test_ds=vm_test_ds)Run SeasonalDecomposeMetricWithFigure Test
# test_context = TestContext(train_ds=vm_train_ds)
# sd_metric = SeasonalDecomposeMetricWithFigure(test_context=test_context)
# sd_metric.run()Run ResidualsVisualInspection Test
test_context = TestContext(train_ds=vm_train_ds, test_ds=vm_test_ds)
rvi_test = ResidualsVisualInspection(test_context=test_context)
rvi_test.run()vm.run_test_plan("seasonality_test_plan", train_ds=vm_train_ds, test_ds=vm_test_ds)loan_rate_columns = ["loan_rate_A", "loan_rate_B", "loan_rate_C", "loan_rate_D"]
diff1_loan_rate_columns = ["diff1_loan_rate_A", "diff1_loan_rate_B", "diff1_loan_rate_C", "diff1_loan_rate_D"]
test_plan_config = {
"time_series_univariate_inspection_raw": {
"columns": loan_rate_columns + diff1_loan_rate_columns
},
"time_series_univariate_inspection_histogram": {
"columns": loan_rate_columns + diff1_loan_rate_columns
}
}
vm.run_test_plan(
"timeseries_test_plan",
config=test_plan_config,
test_ds=vm_test_ds,
train_ds=vm_train_ds,
dataset=vm_train_ds,
)Running Metric: time_series_univariate_inspection_raw: 100%|██████████| 2/2 [00:01<00:00, 1.80it/s]